{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAABECAYAAACmjMM7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAACPElEQVR4nO3ZMYoTUQDG8ZdNSAJZdr1FzmApeBl7QVAP4DE8glhoIWxn4RFi5xViwJhMMhaxy2ZB/F7eEn6/dhj4qvnPmxn0fV8A4H9dtR4AwGUQFAAiBAWACEEBIGJ06sJ8Pu/KITjL880B4JG7KaXsF4vFUT9OBqUcYjKYTsa31WY1tutaL6irG+1bT6jqerBpPaGa1W7WekJV3XrVekJV09mk9YRqtuuulBNftx4KynI6Gd++fvWiyqjH4PuXyz583T3/0XpCVXdP3reeUM3Tr59bT6jq27tnrSdU9ebjy9YTqvnw9lPZrrt7H57+oQAQISgARAgKABGCAkCEoAAQISgARAgKABGCAkCEoAAQISgARAgKABGCAkCEoAAQISgARAgKABGCAkCEoAAQISgARAgKABGCAkCEoAAQISgARAgKABGCAkCEoAAQISgARAgKABGCAkCEoAAQISgARAgKABGCAkCEoAAQISgARAgKABGCAkCEoAAQISgARAgKABGCAkCEoAAQISgARAgKABGCAkCEoAAQISgARAgKABGCAkCEoAAQISgARAgKABGCAkDEoO/7ey/M5/N9KWUwnYzPu+iMdl3rBXV1o33rCVVdDzatJ1Sz2s1aT6iqW69aT6hqOpu0nlDNdt2VUkq/WCyODiSjB+7bl1Ku1r83y1rDqGzXekBdPy/6gP2r9YCqhsNh6wlV/X3oXqqbcujDkZMnFAD4F5f8igfAGQkKABGCAkCEoAAQISgARPwB8K5NVoTYsCUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 504x72 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "import matplotlib.ticker as ticker\n",
    "import statsmodels.formula.api as smf\n",
    "from pprint import pprint\n",
    "import re as re\n",
    "\n",
    "from statsmodels.regression.mixed_linear_model import MixedLMResults\n",
    "\n",
    "import scipy as sp\n",
    "#from scipy.stats import nanmean\n",
    "#from scipy.stats import nanstd\n",
    "import copy\n",
    "import scipy.stats as stats\n",
    "import string\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "from pprint import pprint\n",
    "import seaborn as sns\n",
    "sns.set(style=\"white\", context=\"talk\")\n",
    "custom_palette = [ \"greyish\", \"dusty purple\", \"greenish\", \"amber\", \"windows blue\", \"black\",\"faded green\"]  \n",
    "                     #\"green blue\", \"dull green\", \"faded green\",  \n",
    "sns.set_palette(sns.xkcd_palette(custom_palette))\n",
    "current_palette = sns.color_palette()\n",
    "%matplotlib inline\n",
    "sns.palplot(current_palette)\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "round_data = pd.read_csv('./all_studies_round_data.csv')\n",
    "\n",
    "round_data['group'] = 1 #this is only for the crossed mixed effect model\n",
    "\n",
    "game_rounds = pd.read_csv('./all_studies_game_data.csv')\n",
    "#round_data = round_data[round_data.round_index > 3]\n",
    "#print(round_data.size)\n",
    "conditions = ['dynamic_full_feedback',\n",
    "              'dynamic',         \n",
    "              'dynamic_self_feedback',\n",
    "              'dynamic_no_feedback',\n",
    "              'static',\n",
    "              'solo_feedback',\n",
    "              'solo_no_feedback'\n",
    "             ]\n",
    "\n",
    "\n",
    "colors ={'dynamic_no_feedback':'#196FFF',\n",
    "         'dynamic_self_feedback':'#000000',\n",
    "         'dynamic_full_feedback':'#E89468',\n",
    "         'dynamic': '#81B200',\n",
    "         'static':'#9B59B6',\n",
    "         'solo_feedback': '#95A5A6',\n",
    "         'solo_no_feedback': '#95A5A6'\n",
    "        }\n",
    "\n",
    "\n",
    "linestyles ={'dynamic_no_feedback':'-.',\n",
    "         'dynamic_self_feedback':':',\n",
    "         'dynamic_full_feedback':'-',\n",
    "         'dynamic': '-',\n",
    "         'static':'--',\n",
    "         'solo_feedback': '-',\n",
    "         'solo_no_feedback': '-'\n",
    "        }\n",
    "\n",
    "markers ={'dynamic_no_feedback':'d',\n",
    "         'dynamic_self_feedback':'^',\n",
    "         'dynamic_full_feedback':'p',\n",
    "         'dynamic': 'o',\n",
    "         'static':'*',\n",
    "         'solo_feedback': 'H',\n",
    "         'solo_no_feedback': 'h'\n",
    "        }\n",
    "\n",
    "#markers = ['s','o','*']\n",
    "#linestyles = ['--','-',':','-.']\n",
    "tick_size = 25\n",
    "label_size = 35\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "studies = [1,2]\n",
    "#crossed model with no independent groups https://stackoverflow.com/questions/50052421/mixed-models-with-two-random-effects-statsmodels\n",
    "# this is used as a variance component for the individual level stuff\n",
    "vcf = {\"player_id\": \"0 + C(player_id)\", \"game_id\": \"0 + C(game_id)\"} \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Statistical Tests for study 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table S2: Solo vs dynamic no feedback (individual)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "solo vs dynamic no feedback (individual) : over all\n",
      " β:  -0.04799017966371462 , t-statistic -4.5404985355881875 , pvalue  5.612136869274955e-06  ci  -0.06870575153655138 -0.027274607790877856 reduced error by  -23.073262303358728\n",
      "=============================\n",
      "solo vs dynamic no feedback: first half\n",
      " β:  -0.04227247084347075 , t-statistic -3.6661175350172224 , pvalue  0.00024626077490999326  ci  -0.06487199734510349 -0.019672944341838017 reduced error by  -21.223944051907946\n",
      "=============================\n",
      "solo vs dynamic no feedback (individual): second half\n",
      " β:  -0.05389885302728884 , t-statistic -3.9083385521793015 , pvalue  9.293301816955817e-05  ci  -0.08092819281736376 -0.026869513237213923 reduced error by  -24.734773392824636\n",
      "=============================\n",
      "solo vs dynamic no feedback (individual): adapted periods\n",
      " β:  -0.04160410261002409 , t-statistic -4.101598197967833 , pvalue  4.103063109049347e-05  ci  -0.06148477809041535 -0.02172342712963283 reduced error by  -20.23231645088945\n"
     ]
    }
   ],
   "source": [
    "#### Study 2 Solo vs Dynamic no feedback\n",
    "\n",
    "print(\"solo vs dynamic no feedback (individual) : over all\")\n",
    "\n",
    "forumla = \"revised_error ~ C(condition, Treatment('solo_no_feedback'))\"\n",
    "data = round_data[(round_data.study==2) &\n",
    "                  (~round_data.condition.isin(['dynamic_full_feedback','dynamic_self_feedback'])) &\n",
    "                  (round_data.revised_error.notnull())]\n",
    "model = smf.mixedlm(forumla,data=data,groups='group',\n",
    "                    vc_formula=vcf,re_formula=\"0\").fit(method='powell')\n",
    "\n",
    "\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_no_feedback'].revised_error.mean() - data[data.condition == 'solo_no_feedback'].revised_error.mean() ) / data[data.condition == 'dynamic_no_feedback'].revised_error.mean()\n",
    "\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')\n",
    "#print(model.summary())\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "\n",
    "print(\"solo vs dynamic no feedback: first half\")\n",
    "\n",
    "forumla = \"revised_error ~ C(condition, Treatment('solo_no_feedback'))\"\n",
    "data = round_data[(round_data.study==2) &\n",
    "                  (~round_data.condition.isin(['dynamic_full_feedback','dynamic_self_feedback'])) &\n",
    "                  (round_data.revised_error.notnull()) &\n",
    "                  (round_data.half==1)\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups='group',\n",
    "                    vc_formula=vcf,re_formula=\"0\").fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_no_feedback'].revised_error.mean() - data[data.condition == 'solo_no_feedback'].revised_error.mean() ) / data[data.condition == 'dynamic_no_feedback'].revised_error.mean()\n",
    "\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')\n",
    "#print(model.summary())\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "print(\"solo vs dynamic no feedback (individual): second half\")\n",
    "\n",
    "forumla = \"revised_error ~ C(condition, Treatment('solo_no_feedback'))\"\n",
    "data = round_data[(round_data.study==2) &\n",
    "                  (~round_data.condition.isin(['dynamic_full_feedback','dynamic_self_feedback'])) &\n",
    "                  (round_data.revised_error.notnull()) &\n",
    "                  (round_data.half==2)\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups='group',\n",
    "                    vc_formula=vcf,re_formula=\"0\").fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_no_feedback'].revised_error.mean() - data[data.condition == 'solo_no_feedback'].revised_error.mean() ) / data[data.condition == 'dynamic_no_feedback'].revised_error.mean()\n",
    "\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')\n",
    "#print(model.summary())\n",
    "\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "print(\"solo vs dynamic no feedback (individual): adapted periods\")\n",
    "forumla = \"revised_error ~ C(condition, Treatment('solo_no_feedback'))\"\n",
    "data = round_data[(round_data.study==2) &\n",
    "                  (~round_data.condition.isin(['dynamic_full_feedback','dynamic_self_feedback'])) &\n",
    "                  (round_data.revised_error.notnull()) &\n",
    "                  (round_data.quarter.isin([2,4]))\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups='group',\n",
    "                    vc_formula=vcf,re_formula=\"0\").fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_no_feedback'].revised_error.mean() - data[data.condition == 'solo_no_feedback'].revised_error.mean() ) / data[data.condition == 'dynamic_no_feedback'].revised_error.mean()\n",
    "\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table S2: Solo vs dynamic no feedback (group)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "solo vs dynamic no feedback (group): over all\n",
      " β:  -0.03101231734006802 , t-statistic -4.547534873643473 , pvalue  5.427792846000033e-06  ci  -0.044378465539600384 -0.01764616914053566 reduced error by  -17.52603410006682\n",
      "=============================\n",
      "solo vs dynamic no feedback (group): first half\n",
      " β:  -0.02613844612794651 , t-statistic -2.984259778623684 , pvalue  0.002842653143576798  ci  -0.04330532060368087 -0.00897157165221215 reduced error by  -15.538204525125018\n",
      "=============================\n",
      "solo vs dynamic no feedback (group): second half\n",
      " β:  -0.03588618855218838 , t-statistic -3.6221895700900344 , pvalue  0.00029211991565651646  ci  -0.055304177518170225 -0.016468199586206534 reduced error by  -19.326952909877825\n",
      "=============================\n",
      "solo vs dynamic no feedback (group): adapted periods\n",
      " β:  -0.024547976430976462 , t-statistic -2.47982976004466 , pvalue  0.01314451236257879  ci  -0.04394977185792766 -0.005146181004025269 reduced error by  -14.193808319559384\n"
     ]
    }
   ],
   "source": [
    "#### Study 2 Solo vs Dynamic no feedback\n",
    "\n",
    "print(\"solo vs dynamic no feedback (group): over all\")\n",
    "\n",
    "forumla = \"revised_wisdom_error ~ C(condition, Treatment('solo_no_feedback'))\"\n",
    "data = game_rounds[(game_rounds.study==2) &\n",
    "                  (~game_rounds.condition.isin(['dynamic_full_feedback','dynamic_self_feedback']))\n",
    "                  ]\n",
    "model = smf.mixedlm(forumla,data=data,groups=data['game_id']).fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_no_feedback'].revised_wisdom_error.mean() - data[data.condition == 'solo_no_feedback'].revised_wisdom_error.mean() ) / data[data.condition == 'dynamic_no_feedback'].revised_wisdom_error.mean()\n",
    "\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "\n",
    "print(\"solo vs dynamic no feedback (group): first half\")\n",
    "\n",
    "forumla = \"revised_wisdom_error ~ C(condition, Treatment('solo_no_feedback'))\"\n",
    "data = game_rounds[(game_rounds.study==2) &\n",
    "                  (~game_rounds.condition.isin(['dynamic_full_feedback','dynamic_self_feedback'])) &\n",
    "                  (game_rounds.half==1)\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups=data['game_id']).fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_no_feedback'].revised_wisdom_error.mean() - data[data.condition == 'solo_no_feedback'].revised_wisdom_error.mean() ) / data[data.condition == 'dynamic_no_feedback'].revised_wisdom_error.mean()\n",
    "\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())\n",
    "\n",
    "#print(model.summary())\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "print(\"solo vs dynamic no feedback (group): second half\")\n",
    "\n",
    "forumla = \"revised_wisdom_error ~ C(condition, Treatment('solo_no_feedback'))\"\n",
    "data = game_rounds[(game_rounds.study==2) &\n",
    "                  (~game_rounds.condition.isin(['dynamic_full_feedback','dynamic_self_feedback'])) &\n",
    "                  (game_rounds.half==2)\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups=data['game_id']).fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_no_feedback'].revised_wisdom_error.mean() - data[data.condition == 'solo_no_feedback'].revised_wisdom_error.mean() ) / data[data.condition == 'dynamic_no_feedback'].revised_wisdom_error.mean()\n",
    "\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())\n",
    "\n",
    "#print(model.summary())\n",
    "\n",
    "\n",
    "print('=============================')\n",
    "print(\"solo vs dynamic no feedback (group): adapted periods\")\n",
    "\n",
    "forumla = \"revised_wisdom_error ~ C(condition, Treatment('solo_no_feedback'))\"\n",
    "data = game_rounds[(game_rounds.study==2) &\n",
    "                  (~game_rounds.condition.isin(['dynamic_full_feedback','dynamic_self_feedback'])) &\n",
    "                  (game_rounds.quarter.isin([2,4]))\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups=data['game_id']).fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_no_feedback'].revised_wisdom_error.mean() - data[data.condition == 'solo_no_feedback'].revised_wisdom_error.mean() ) / data[data.condition == 'dynamic_no_feedback'].revised_wisdom_error.mean()\n",
    "\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Table S2:  dynamic no feedback vs dynamic self feedback (individual)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dynamic no feedback vs dynamic self feedback (individual): over all\n",
      " β:  -0.014501775840543359 , t-statistic -1.3771712490407815 , pvalue  0.1684593085852789  ci  -0.03514042799011084 0.006136876309024122 reduced error by  -7.381813654599785\n",
      "=============================\n",
      "dynamic no feedback vs dynamic self feedback (individual): first half\n",
      " β:  -0.010516261456367628 , t-statistic -0.87179762006812 , pvalue  0.3833187931766716  ci  -0.03415878264709623 0.013126259734360974 reduced error by  -5.548284690254318\n",
      "=============================\n",
      "dynamic no feedback vs dynamic self feedback (individual): second half\n",
      " β:  -0.01838964749226939 , t-statistic -1.3191731646205391 , pvalue  0.1871112275632718  ci  -0.04571210048016286 0.008932805495624078 reduced error by  -9.130284595958415\n",
      "=============================\n",
      "dynamic no feedback vs dynamic self feedback (individual): adapted periods\n",
      " β:  -0.03677095684431783 , t-statistic -3.524075258796028 , pvalue  0.000424963528870179  ci  -0.05722164129325369 -0.01632027239538197 reduced error by  -21.628549556967002\n"
     ]
    }
   ],
   "source": [
    "#### Study 2 dynamic no fedback vs dynamic self feedback\n",
    "\n",
    "print(\"dynamic no feedback vs dynamic self feedback (individual): over all\")\n",
    "\n",
    "forumla = \"revised_error ~ C(condition, Treatment('dynamic_no_feedback'))\"\n",
    "data = round_data[(round_data.study==2) &\n",
    "                  (~round_data.condition.isin(['dynamic_full_feedback','solo_no_feedback'])) &\n",
    "                  (round_data.revised_error.notnull())]\n",
    "model = smf.mixedlm(forumla,data=data,groups='group',\n",
    "                    vc_formula=vcf,re_formula=\"0\").fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_self_feedback'].revised_error.mean() - data[data.condition == 'dynamic_no_feedback'].revised_error.mean() ) / data[data.condition == 'dynamic_self_feedback'].revised_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "\n",
    "print(\"dynamic no feedback vs dynamic self feedback (individual): first half\")\n",
    "\n",
    "forumla = \"revised_error ~ C(condition, Treatment('dynamic_no_feedback'))\"\n",
    "data = round_data[(round_data.study==2) &\n",
    "                  (~round_data.condition.isin(['dynamic_full_feedback','solo_no_feedback'])) &\n",
    "                  (round_data.revised_error.notnull()) &\n",
    "                  (round_data.half==1)\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups='group',\n",
    "                    vc_formula=vcf,re_formula=\"0\").fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_self_feedback'].revised_error.mean() - data[data.condition == 'dynamic_no_feedback'].revised_error.mean() ) / data[data.condition == 'dynamic_self_feedback'].revised_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())#print(model.summary())\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "print(\"dynamic no feedback vs dynamic self feedback (individual): second half\")\n",
    "\n",
    "forumla = \"revised_error ~ C(condition, Treatment('dynamic_no_feedback'))\"\n",
    "data = round_data[(round_data.study==2) &\n",
    "                  (~round_data.condition.isin(['dynamic_full_feedback','solo_no_feedback'])) &\n",
    "                  (round_data.revised_error.notnull()) &\n",
    "                  (round_data.half==2)\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups='group',\n",
    "                    vc_formula=vcf,re_formula=\"0\").fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_self_feedback'].revised_error.mean() - data[data.condition == 'dynamic_no_feedback'].revised_error.mean() ) / data[data.condition == 'dynamic_self_feedback'].revised_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())#print(model.summary())\n",
    "\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "print(\"dynamic no feedback vs dynamic self feedback (individual): adapted periods\")\n",
    "forumla = \"revised_error ~ C(condition, Treatment('dynamic_no_feedback'))\"\n",
    "data = round_data[(round_data.study==2) &\n",
    "                  (~round_data.condition.isin(['dynamic_full_feedback','solo_no_feedback'])) &\n",
    "                  (round_data.revised_error.notnull()) &\n",
    "                  (round_data.quarter.isin([2,4]))\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups='group',\n",
    "                    vc_formula=vcf,re_formula=\"0\").fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_self_feedback'].revised_error.mean() - data[data.condition == 'dynamic_no_feedback'].revised_error.mean() ) / data[data.condition == 'dynamic_self_feedback'].revised_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Table S2:  dynamic no feedback vs dynamic self feedback (group)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dynamic no feedback vs dynamic self feedback (group): over all\n",
      " β:  -0.018645707070706775 , t-statistic -1.6649849224524635 , pvalue  0.09591576394383117  ci  -0.04059480332500784 0.003303389183594286 reduced error by  -11.778396356588527\n",
      "=============================\n",
      "dynamic no feedback vs dynamic self feedback (group): first half\n",
      " β:  -0.011440707070707028 , t-statistic -0.9335775539406351 , pvalue  0.35052184822441046  ci  -0.03545946557835254 0.012578051436938485 reduced error by  -7.2973094863798496\n",
      "=============================\n",
      "dynamic no feedback vs dynamic self feedback (group): second half\n",
      " β:  -0.025850707070706976 , t-statistic -1.6561833541182212 , pvalue  0.09768469316826806  ci  -0.05644300514416288 0.004741591002748929 reduced error by  -16.17399932377144\n",
      "=============================\n",
      "dynamic no feedback vs dynamic self feedback (group): adapted periods\n",
      " β:  -0.03835050505050507 , t-statistic -2.935087846191384 , pvalue  0.00333453270609237  ci  -0.06395982668816831 -0.012741183412841837 reduced error by  -28.492630504607813\n"
     ]
    }
   ],
   "source": [
    "#### Study 2 dynamic no fedback vs dynamic self feedback\n",
    "\n",
    "print(\"dynamic no feedback vs dynamic self feedback (group): over all\")\n",
    "\n",
    "forumla = \"revised_wisdom_error ~ C(condition, Treatment('dynamic_no_feedback'))\"\n",
    "data = game_rounds[(game_rounds.study==2) &\n",
    "                  (~game_rounds.condition.isin(['dynamic_full_feedback','solo_no_feedback']))]\n",
    "model = smf.mixedlm(forumla,data=data,groups=data['game_id']).fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_self_feedback'].revised_wisdom_error.mean() - data[data.condition == 'dynamic_no_feedback'].revised_wisdom_error.mean() ) / data[data.condition == 'dynamic_self_feedback'].revised_wisdom_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())#print(model.summary())\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "\n",
    "print(\"dynamic no feedback vs dynamic self feedback (group): first half\")\n",
    "\n",
    "forumla = \"revised_wisdom_error ~ C(condition, Treatment('dynamic_no_feedback'))\"\n",
    "data = game_rounds[(game_rounds.study==2) &\n",
    "                  (~game_rounds.condition.isin(['dynamic_full_feedback','solo_no_feedback'])) &\n",
    "                  (game_rounds.half==1)\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups=data['game_id']).fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_self_feedback'].revised_wisdom_error.mean() - data[data.condition == 'dynamic_no_feedback'].revised_wisdom_error.mean() ) / data[data.condition == 'dynamic_self_feedback'].revised_wisdom_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())#print(model.summary())\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "print(\"dynamic no feedback vs dynamic self feedback (group): second half\")\n",
    "\n",
    "forumla = \"revised_wisdom_error ~ C(condition, Treatment('dynamic_no_feedback'))\"\n",
    "data = game_rounds[(game_rounds.study==2) &\n",
    "                  (~game_rounds.condition.isin(['dynamic_full_feedback','solo_no_feedback'])) &\n",
    "                  (game_rounds.half==2)\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups=data['game_id']).fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_self_feedback'].revised_wisdom_error.mean() - data[data.condition == 'dynamic_no_feedback'].revised_wisdom_error.mean() ) / data[data.condition == 'dynamic_self_feedback'].revised_wisdom_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())#print(model.summary())\n",
    "\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "print(\"dynamic no feedback vs dynamic self feedback (group): adapted periods\")\n",
    "forumla = \"revised_wisdom_error ~ C(condition, Treatment('dynamic_no_feedback'))\"\n",
    "data = game_rounds[(game_rounds.study==2) &\n",
    "                  (~game_rounds.condition.isin(['dynamic_full_feedback','solo_no_feedback'])) &\n",
    "                  (game_rounds.quarter.isin([2,4]))\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups=data['game_id']).fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_self_feedback'].revised_wisdom_error.mean() - data[data.condition == 'dynamic_no_feedback'].revised_wisdom_error.mean() ) / data[data.condition == 'dynamic_self_feedback'].revised_wisdom_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Table S2:  dynamic self feedback vs dynamic full feedback (individual)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dynamic self feedback vs dynamic full feedback (individual): over all\n",
      " β:  -0.03200019432749047 , t-statistic -3.294833493746869 , pvalue  0.0009848001315186  ci  -0.05103582344173753 -0.01296456521324341 reduced error by  -19.76390644678898\n",
      "=============================\n",
      "dynamic self feedback vs dynamic full feedback (individual): first half\n",
      " β:  -0.036377930444661026 , t-statistic -3.6363492387637915 , pvalue  0.00027652936366632413  ci  -0.05598535229061046 -0.01677050859871159 reduced error by  -23.717570878590678\n",
      "=============================\n",
      "dynamic self feedback vs dynamic full feedback (individual): second half\n",
      " β:  -0.027571074806374635 , t-statistic -1.88761864200954 , pvalue  0.05907716558741152  ci  -0.05619884549740006 0.0010566958846507934 reduced error by  -16.24386895113932\n",
      "=============================\n",
      "dynamic self feedback vs dynamic full feedback (individual): adapted periods\n",
      " β:  -0.041490550185891435 , t-statistic -4.988130755655522 , pvalue  6.096630168073345e-07  ci  -0.05779324713722908 -0.025187853234553786 reduced error by  -32.517701639164684\n"
     ]
    }
   ],
   "source": [
    "#### Study 1 Solo vs Static\n",
    "\n",
    "print(\"dynamic self feedback vs dynamic full feedback (individual): over all\")\n",
    "\n",
    "forumla = \"revised_error ~ C(condition, Treatment('dynamic_self_feedback'))\"\n",
    "data = round_data[(round_data.study==2) &\n",
    "                  (~round_data.condition.isin(['dynamic_no_feedback','solo_no_feedback'])) &\n",
    "                  (round_data.revised_error.notnull())]\n",
    "model = smf.mixedlm(forumla,data=data,groups='group',\n",
    "                    vc_formula=vcf,re_formula=\"0\").fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_full_feedback'].revised_error.mean() - data[data.condition == 'dynamic_self_feedback'].revised_error.mean() ) / data[data.condition == 'dynamic_full_feedback'].revised_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())#print(model.summary())\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "\n",
    "print(\"dynamic self feedback vs dynamic full feedback (individual): first half\")\n",
    "\n",
    "forumla = \"revised_error ~ C(condition, Treatment('dynamic_self_feedback'))\"\n",
    "data = round_data[(round_data.study==2) &\n",
    "                  (~round_data.condition.isin(['dynamic_no_feedback','solo_no_feedback'])) &\n",
    "                  (round_data.revised_error.notnull()) &\n",
    "                  (round_data.half==1)\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups='group',\n",
    "                    vc_formula=vcf,re_formula=\"0\").fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_full_feedback'].revised_error.mean() - data[data.condition == 'dynamic_self_feedback'].revised_error.mean() ) / data[data.condition == 'dynamic_full_feedback'].revised_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())#print(model.summary())\n",
    "#print(model.summary())\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "print(\"dynamic self feedback vs dynamic full feedback (individual): second half\")\n",
    "\n",
    "forumla = \"revised_error ~ C(condition, Treatment('dynamic_self_feedback'))\"\n",
    "data = round_data[(round_data.study==2) &\n",
    "                  (~round_data.condition.isin(['dynamic_no_feedback','solo_no_feedback'])) &\n",
    "                  (round_data.revised_error.notnull()) &\n",
    "                  (round_data.half==2)\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups='group',\n",
    "                    vc_formula=vcf,re_formula=\"0\").fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_full_feedback'].revised_error.mean() - data[data.condition == 'dynamic_self_feedback'].revised_error.mean() ) / data[data.condition == 'dynamic_full_feedback'].revised_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())#print(model.summary())\n",
    "#print(model.summary())\n",
    "\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "print(\"dynamic self feedback vs dynamic full feedback (individual): adapted periods\")\n",
    "forumla = \"revised_error ~ C(condition, Treatment('dynamic_self_feedback'))\"\n",
    "data = round_data[(round_data.study==2) &\n",
    "                  (~round_data.condition.isin(['dynamic_no_feedback','solo_no_feedback'])) &\n",
    "                  (round_data.revised_error.notnull()) &\n",
    "                  (round_data.quarter.isin([2,4]))\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups='group',\n",
    "                    vc_formula=vcf,re_formula=\"0\").fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_full_feedback'].revised_error.mean() - data[data.condition == 'dynamic_self_feedback'].revised_error.mean() ) / data[data.condition == 'dynamic_full_feedback'].revised_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())#print(model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Table S2:  dynamic self feedback vs dynamic full feedback (group)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dynamic self feedback vs dynamic full feedback (group): over all\n",
      " β:  -0.021740435305434968 , t-statistic -2.2520838189830874 , pvalue  0.024316975645446635  ci  -0.0406608990322762 -0.002819971578593733 reduced error by  -15.919611296654832\n",
      "=============================\n",
      "dynamic self feedback vs dynamic full feedback (group): first half\n",
      " β:  -0.03144639249639255 , t-statistic -2.8608923322238446 , pvalue  0.004224504928631609  ci  -0.05298994939380661 -0.00990283559897849 reduced error by  -25.090192335479223\n",
      "=============================\n",
      "dynamic self feedback vs dynamic full feedback (group): second half\n",
      " β:  -0.012034478114478131 , t-statistic -0.7665905243346921 , pvalue  0.44332501106832445  ci  -0.042803373539559764 0.018734417310603498 reduced error by  -8.142720875837133\n",
      "=============================\n",
      "dynamic self feedback vs dynamic full feedback (group): adapted periods\n",
      " β:  -0.02993710437710449 , t-statistic -2.9916917596855948 , pvalue  0.0027743623456864255  ci  -0.04954996930230239 -0.010324239451906584 reduced error by  -28.60391168783709\n"
     ]
    }
   ],
   "source": [
    "#### Study 2 dynamic self feedback vs dynamic full feedback\n",
    "\n",
    "print(\"dynamic self feedback vs dynamic full feedback (group): over all\")\n",
    "forumla = \"revised_wisdom_error ~ C(condition, Treatment('dynamic_self_feedback'))\"\n",
    "data = game_rounds[(game_rounds.study==2) &\n",
    "                  (~game_rounds.condition.isin(['dynamic_no_feedback','solo_no_feedback']))]\n",
    "model = smf.mixedlm(forumla,data=data,groups=data['game_id']).fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_full_feedback'].revised_wisdom_error.mean() - data[data.condition == 'dynamic_self_feedback'].revised_wisdom_error.mean() ) / data[data.condition == 'dynamic_full_feedback'].revised_wisdom_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())#print(model.summary())#print(model.summary())\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "\n",
    "print(\"dynamic self feedback vs dynamic full feedback (group): first half\")\n",
    "\n",
    "forumla = \"revised_wisdom_error ~ C(condition, Treatment('dynamic_self_feedback'))\"\n",
    "data = game_rounds[(game_rounds.study==2) &\n",
    "                  (~game_rounds.condition.isin(['dynamic_no_feedback','solo_no_feedback'])) &\n",
    "                  (game_rounds.half==1)\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups=data['game_id']).fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_full_feedback'].revised_wisdom_error.mean() - data[data.condition == 'dynamic_self_feedback'].revised_wisdom_error.mean() ) / data[data.condition == 'dynamic_full_feedback'].revised_wisdom_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())#print(model.summary())#print(model.summary())\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "print(\"dynamic self feedback vs dynamic full feedback (group): second half\")\n",
    "\n",
    "forumla = \"revised_wisdom_error ~ C(condition, Treatment('dynamic_self_feedback'))\"\n",
    "data = game_rounds[(game_rounds.study==2) &\n",
    "                  (~game_rounds.condition.isin(['dynamic_no_feedback','solo_no_feedback'])) &\n",
    "                  (game_rounds.half==2)\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups=data['game_id']).fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_full_feedback'].revised_wisdom_error.mean() - data[data.condition == 'dynamic_self_feedback'].revised_wisdom_error.mean() ) / data[data.condition == 'dynamic_full_feedback'].revised_wisdom_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())#print(model.summary())#print(model.summary())\n",
    "\n",
    "\n",
    "print('=============================')\n",
    "\n",
    "print(\"dynamic self feedback vs dynamic full feedback (group): adapted periods\")\n",
    "forumla = \"revised_wisdom_error ~ C(condition, Treatment('dynamic_self_feedback'))\"\n",
    "data = game_rounds[(game_rounds.study==2) &\n",
    "                  (~game_rounds.condition.isin(['dynamic_no_feedback','solo_no_feedback'])) &\n",
    "                  (game_rounds.quarter.isin([2,4]))\n",
    "                 ]\n",
    "model = smf.mixedlm(forumla,data=data,groups=data['game_id']).fit(method='powell')\n",
    "\n",
    "reduced_error = (data[data.condition == 'dynamic_full_feedback'].revised_wisdom_error.mean() - data[data.condition == 'dynamic_self_feedback'].revised_wisdom_error.mean() ) / data[data.condition == 'dynamic_full_feedback'].revised_wisdom_error.mean()\n",
    "print(\" β: \",model.params[1], ', t-statistic',model.tvalues[1], ', pvalue ', model.pvalues[1], \n",
    "      ' ci ', model.conf_int()[0][1], model.conf_int()[1][1], 'reduced error by ', reduced_error*100)#print (' ---- ')#print(model.summary())#print(model.summary())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
